## =======================================================================================
##
##   Command line script for the sequence analysis as presented
##   Hu et al. (2018) in Nature Communications
## 
##  |  2018-05-30
## 
##   Klaus Schlaeppi, klaus.schlaeppi@ips.unibe.ch 
##
## =======================================================================================






## =======================================================================================
## =======================================================================================
## prep | environment
## =======================================================================================
## =======================================================================================


#### MiSeq run #2 | 16S
## =======================================================================================
## The field experiment was sequenced in the MiSeq run #2.
## This MiSeq library contains additional samples from experiments 
## that are not related to this study. They were presented in Hartman et al. (2017) in Microbiome.
## The raw data of MiSeq run #2 was stored at ENA under the study accession PRJEB15152 (Sample: SAMEA54297418).

/yourpath/MiSeq_run02/
mkdir a_data
mkdir a_data/gz

# /yourpath/MiSeq_run02/a_data/gz/ with:
s1-amplicon_S1_L001_R1_001.fastq.gz
s1-amplicon_S1_L001_R2_001.fastq.gz


#### MiSeq run #8 | 16S
## =======================================================================================
## The feedback experiment was sequenced in the MiSeq run #8
## This MiSeq library contain additional samples from experiments (publication in prep.)
## that are not related to this study. The raw data of the MiSeq run #8
## is stored at at ENA under the study accession PRJEB20127 (Sample: SAMEA103939171).

/yourpath/MiSeq_run08/
mkdir a_data
mkdir a_data/gz

# /yourpath/MiSeq_run02/a_data/gz/ with:
run8_S1_L001_R1_001.fastq.gz
run8_S1_L001_R2_001.fastq.gz


#### MiSeq run #11 | ITS
## =======================================================================================
## Both field and feedback experiments were sequenced in the MiSeq run #11.
## This MiSeq library also contains additional samples from experiments 
## that are not related to this study. The raw data of the MiSeq run #11
## is stored together with MiSeq run #8 at ENA under the study accession PRJEB20127 (Sample: SAMEA4698767).

/yourpath/MiSeq_run11/
mkdir a_data
mkdir a_data/gz

# /yourpath/MiSeq_run11/a_data/gz/ with:
p1617-3554-01_S1_L001_R1_001.fastq.gz
p1617-3554-01_S1_L001_R2_001.fastq.gz






## =======================================================================================
## =======================================================================================
## A | QC and gz2fq
## =======================================================================================
## =======================================================================================

## ---------------------------------------------------------------------------------------
## A1 | QC - FastQC v0.11.2
## ---------------------------------------------------------------------------------------

mkdir MiSeq_run02/a_data/qc/
fastqc -t 20 -k 8 -q MiSeq_run02/a_data/gz/s1-amplicon_S1_L001_R1_001.fastq.gz MiSeq_run02/a_data/gz/s1-amplicon_S1_L001_R2_001.fastq.gz -o MiSeq_run02/a_data/qc/

mkdir MiSeq_run08/a_data/qc/
fastqc -t 20 -k 8 -q MiSeq_run08/a_data/gz/run8_S1_L001_R1_001.fastq.gz MiSeq_run08/a_data/gz/run8_S1_L001_R2_001.fastq.gz -o MiSeq_run08/a_data/qc/

mkdir MiSeq_run11/a_data/qc/
fastqc -t 20 -k 8 -q MiSeq_run11/a_data/gz/p1617-3554-01_S1_L001_R1_001.fastq.gz MiSeq_run11/a_data/gz/p1617-3554-01_S1_L001_R2_001.fastq.gz -o MiSeq_run11/a_data/qc/

## -t = number of files to process at once
## -k = kmer size

## disk space
# Remove the zipped fasta file - these files are not needed
rm MiSeq_run02/a_data/qc/*.zip
rm MiSeq_run08/a_data/qc/*.zip
rm MiSeq_run11/a_data/qc/*.zip


## ---------------------------------------------------------------------------------------
## A2 | GZ > FQ
## ---------------------------------------------------------------------------------------

mkdir MiSeq_run02/a_data/fq/
mkdir MiSeq_run08/a_data/fq/
mkdir MiSeq_run11/a_data/fq/

# Unzip the files but keep a copy
gunzip -c MiSeq_run02/a_data/gz/s1-amplicon_S1_L001_R1_001.fastq.gz > MiSeq_run02/a_data/fq/run02_S1_L001_R1_001.fastq &
gunzip -c MiSeq_run02/a_data/gz/s1-amplicon_S1_L001_R2_001.fastq.gz > MiSeq_run02/a_data/fq/run02_S1_L001_R2_001.fastq &

gunzip -c MiSeq_run08/a_data/gz/run8_S1_L001_R1_001.fastq.gz > MiSeq_run08/a_data/fq/run8_S1_L001_R1_001.fastq &
gunzip -c MiSeq_run08/a_data/gz/run8_S1_L001_R2_001.fastq.gz > MiSeq_run08/a_data/fq/run8_S1_L001_R2_001.fastq &

gunzip -c MiSeq_run11/a_data/gz/p1617-3554-01_S1_L001_R1_001.fastq.gz > MiSeq_run11/a_data/fq/run11_S1_L001_R1_001.fastq &
gunzip -c MiSeq_run11/a_data/gz/p1617-3554-01_S1_L001_R2_001.fastq.gz > MiSeq_run11/a_data/fq/run11_S1_L001_R2_001.fastq &




## =======================================================================================
## =======================================================================================
## B | Trim low quality ends
## =======================================================================================
## =======================================================================================

# Problem: The merging of reads with low quality endings is difficult. We trim the end off
#          (-20nt) to improve merging success. Alternatively, it would be possible to only 
#          only trim the reverse primer. 
#          We also remove short reads (<100nt) and reads with more than 1 ambiguous nucleotides.
#          Again, we could be most stringent with the filtering and include qf.    

mkdir MiSeq_run02/b_trim/
prinseq-lite.pl -verbose --out_format 3 -ns_max_n 1 --min_len 100 -trim_to_len 280 -fastq MiSeq_run02/a_data/fq/run02_S1_L001_R1_001.fastq -fastq2 MiSeq_run02/a_data/fq/run02_S1_L001_R2_001.fastq -out_good MiSeq_run02/b_trim/run02_trim -out_bad MiSeq_run02/b_trim/run02_fail -log MiSeq_run02/b_trim/run02.log &

mkdir MiSeq_run08/b_trim/
prinseq-lite.pl -verbose --out_format 3 -ns_max_n 1 --min_len 100 -trim_to_len 280 -fastq MiSeq_run08/a_data/fq/run8_S1_L001_R1_001.fastq -fastq2 MiSeq_run08/a_data/fq/run8_S1_L001_R2_001.fastq -out_good MiSeq_run08/b_trim/run08_trim -out_bad MiSeq_run08/b_trim/run08_fail -log MiSeq_run08/b_trim/run08.log &

mkdir MiSeq_run11/b_trim/
prinseq-lite.pl -verbose --out_format 3 -ns_max_n 1 --min_len 100 -trim_to_len 280 -fastq MiSeq_run11/a_data/fq/run11_S1_L001_R1_001.fastq -fastq2 MiSeq_run11/a_data/fq/run11_S1_L001_R2_001.fastq -out_good MiSeq_run11/b_trim/run11_trim -out_bad MiSeq_run11/b_trim/run11_fail -log MiSeq_run11/b_trim/run11.log &


## -verbose       = print status information during the processing
## --out_format 3 = what kind of file for the output; in this case FASTQ
## -ns_max_n      = remove sequences with 1 N
## --min_len      = filter sequences shorter than 100
## -trim_to_len   = trim all sequences from 3' end to result in a sequence length 280


## disk space
# Remove the fastq files in a_data/fq - these files are not needed any longer
rm MiSeq_run02/a_data/fq/*.fastq
rm MiSeq_run08/a_data/fq/*.fastq
rm MiSeq_run11/a_data/fq/*.fastq




## =======================================================================================
## =======================================================================================
## C | Merge overlap reads - FLASH v1.2.9
## =======================================================================================
## =======================================================================================

mkdir MiSeq_run02/c_merge/
flash MiSeq_run02/b_trim/run02_trim_1.fastq MiSeq_run02/b_trim/run02_trim_2.fastq -t 10 -m 15 -M 250 -x 0.25 -d MiSeq_run02/c_merge/ -o run02 &

mkdir MiSeq_run08/c_merge/
flash MiSeq_run08/b_trim/run08_trim_1.fastq MiSeq_run08/b_trim/run08_trim_2.fastq -t 10 -m 15 -M 250 -x 0.25 -d MiSeq_run08/c_merge/ -o run08 &

mkdir MiSeq_run11/c_merge/
flash MiSeq_run11/b_trim/run11_trim_1.fastq MiSeq_run11/b_trim/run11_trim_2.fastq -t 10 -m 15 -M 250 -x 0.25 -d MiSeq_run11/c_merge/ -o run11 &

# -t = number of threads
# -m = minimum required overlap length to provide a confident overlap
# -M = maximum overlap number
# -x = maximum allowed ratio between number of mismatched BPs and the overlap length
# -d = output directory

# Remove the fastq files in b_trim/ - these files are not needed any longer
rm MiSeq_run02/b_trim/*.fastq
rm MiSeq_run08/b_trim/*.fastq

fastx_reverse_complement -i MiSeq_run02/c_merge/run02.extendedFrags.fastq -o MiSeq_run02/c_merge/run02.extendedFrags_reversed.fastq -Q33 &
cat MiSeq_run02/c_merge/run02.extendedFrags.fastq MiSeq_run02/c_merge/run02.extendedFrags_reversed.fastq > MiSeq_run02/c_merge/run02_doubled.fastq &

fastx_reverse_complement -i MiSeq_run08/c_merge/run08.extendedFrags.fastq -o MiSeq_run08/c_merge/run08.extendedFrags_reversed.fastq -Q33 &
cat MiSeq_run08/c_merge/run08.extendedFrags.fastq MiSeq_run08/c_merge/run08.extendedFrags_reversed.fastq > MiSeq_run08/c_merge/run08_doubled.fastq &

fastx_reverse_complement -i MiSeq_run11/c_merge/run11.extendedFrags.fastq -o MiSeq_run11/c_merge/run11.extendedFrags_reversed.fastq -Q33 &
cat MiSeq_run11/c_merge/run11.extendedFrags.fastq MiSeq_run11/c_merge/run11.extendedFrags_reversed.fastq > MiSeq_run11/c_merge/run11_doubled.fastq &

## disk space
# Remove the fastq files in c_merge/ - these files are not needed any longer
rm MiSeq_run02/c_merge/*extended*.fastq
rm MiSeq_run02/c_merge/*notCombined*.fastq
rm MiSeq_run08/c_merge/*extended*.fastq
rm MiSeq_run08/c_merge/*notCombined*.fastq
rm MiSeq_run11/c_merge/*extended*.fastq
rm MiSeq_run11/c_merge/*notCombined*.fastq

### keep c_merge/*_doubled.fastq data!




## =======================================================================================
## =======================================================================================
## D | Primer Splitting
## =======================================================================================
## =======================================================================================


## =======================================================================================
## D1 | MiSeq run02
## =======================================================================================
## check if "c_merge/run02_doubled.fastq" needs to be unzipped
# gunzip c_merge/run02_doubled.fastq

mkdir MiSeq_run02/d_primer/
cd MiSeq_run02/

## splitting first by R-primer: 
## =======================================================================================
# cutadapt -a ADAPTER-SEQUENCE input.fastq > output.fastq
# -------------------------------------------------------
# -g ADAPTER, --front = ADAPTER Sequence of an adapter that was ligated to the 5' end.
# -a ADAPTER, --adapter=ADAPTER Sequence of an adapter that was ligated to the 3' end.
# -b ADAPTER, --anywhere=ADAPTER Sequence of an adapter that was ligated to the 5' or 3' end. 

## De-multiplex samples using bc and primer sequence
while read BCPR_ID BCPR
do
rm d_primer/run02_trim_${BCPR_ID}.log
touch d_primer/run02_trim_${BCPR_ID}.log
./d_primer/_demultiplex_by_R_primer.sh $BCPR_ID $BCPR >> d_primer/run02_trim_${BCPR_ID}.log
done < d_primer/_R_primers.txt &

### disk space
gzip c_merge/run02_doubled.fastq &

## splitting then by F-primer the separately "R-splitted" files: 
## =======================================================================================
# cutadapt -a ADAPTER-SEQUENCE input.fastq > output.fastq
# -------------------------------------------------------
# -g ADAPTER, --front = ADAPTER Sequence of an adapter that was ligated to the 5' end.
# -a ADAPTER, --adapter=ADAPTER Sequence of an adapter that was ligated to the 3' end.
# -b ADAPTER, --anywhere=ADAPTER Sequence of an adapter that was ligated to the 5' or 3' end. 

## R3: De-multiplex samples using bc and primer sequence
## =======================================================================================
while read BCPF_ID BCPF
do
touch d_primer/run02_trim_R3_1193R_${BCPF_ID}.log
./d_primer/_demultiplex_R3_by_F_primers.sh $BCPF_ID $BCPF >> d_primer/run02_trim_R3_1193R_${BCPF_ID}.log
done < d_primer/_R3_samples.txt &

## R4: De-multiplex samples using bc and primer sequence
## =======================================================================================
while read BCPF_ID BCPF
do
touch d_primer/run02_trim_R4_1193R_${BCPF_ID}.log
./d_primer/_demultiplex_R4_by_F_primers.sh $BCPF_ID $BCPF >> d_primer/run02_trim_R4_1193R_${BCPF_ID}.log
done < d_primer/_R4_samples.txt &

### sequence counts
grep -c "@M01106" d_primer/run02_trim_R*_F*.fastq > d_primer/run02_trim_R_and_F_seq_counts.txt

cd ..




## =======================================================================================
## D2 | MiSeq run08
## =======================================================================================
## check if "c_merge/run08_doubled.fastq" needs to be unzipped
# gunzip c_merge/run08_doubled.fastq

mkdir MiSeq_run08/d_primer/
cd MiSeq_run08/

## splitting first by R-primer: 
## =======================================================================================
# cutadapt -a ADAPTER-SEQUENCE input.fastq > output.fastq
# -------------------------------------------------------
# -g ADAPTER, --front = ADAPTER Sequence of an adapter that was ligated to the 5' end.
# -a ADAPTER, --adapter=ADAPTER Sequence of an adapter that was ligated to the 3' end.
# -b ADAPTER, --anywhere=ADAPTER Sequence of an adapter that was ligated to the 5' or 3' end. 

## De-multiplex samples using bc and primer sequence
while read BCPR_ID BCPR
do
rm d_primer/run08_trim_${BCPR_ID}.log
touch d_primer/run08_trim_${BCPR_ID}.log
./d_primer/_demultiplex_by_R_primer.sh $BCPR_ID $BCPR >> d_primer/run08_trim_${BCPR_ID}.log
done < d_primer/_R_primers.txt &

### disk space
gzip c_merge/run08_doubled.fastq &


## splitting then by F-primer the separately "R-splitted" files: 
## =======================================================================================
# cutadapt -a ADAPTER-SEQUENCE input.fastq > output.fastq
# -------------------------------------------------------
# -g ADAPTER, --front = ADAPTER Sequence of an adapter that was ligated to the 5' end.
# -a ADAPTER, --adapter=ADAPTER Sequence of an adapter that was ligated to the 3' end.
# -b ADAPTER, --anywhere=ADAPTER Sequence of an adapter that was ligated to the 5' or 3' end. 

## R40: De-multiplex samples using bc and primer sequence
## =======================================================================================
while read BCPF_ID BCPF
do
touch d_primer/run08_trim_R40_1193R_${BCPF_ID}.log
./d_primer/_demultiplex_R40_by_F_primers.sh $BCPF_ID $BCPF >> d_primer/run08_trim_R40_1193R_${BCPF_ID}.log
done < d_primer/_R40_samples.txt &

## R41: De-multiplex samples using bc and primer sequence
## =======================================================================================
while read BCPF_ID BCPF
do
touch d_primer/run08_trim_R41_1193R_${BCPF_ID}.log
./d_primer/_demultiplex_R41_by_F_primers.sh $BCPF_ID $BCPF >> d_primer/run08_trim_R41_1193R_${BCPF_ID}.log
done < d_primer/_R41_samples.txt &

## R42: De-multiplex samples using bc and primer sequence
## =======================================================================================
while read BCPF_ID BCPF
do
touch d_primer/run08_trim_R42_1193R_${BCPF_ID}.log
./d_primer/_demultiplex_R42_by_F_primers.sh $BCPF_ID $BCPF >> d_primer/run08_trim_R42_1193R_${BCPF_ID}.log
done < d_primer/_R42_samples.txt &

## R43: De-multiplex samples using bc and primer sequence
## =======================================================================================
while read BCPF_ID BCPF
do
touch d_primer/run08_trim_R43_1193R_${BCPF_ID}.log
./d_primer/_demultiplex_R43_by_F_primers.sh $BCPF_ID $BCPF >> d_primer/run08_trim_R43_1193R_${BCPF_ID}.log
done < d_primer/_R43_samples.txt &

## R44: De-multiplex samples using bc and primer sequence
## =======================================================================================
while read BCPF_ID BCPF
do
touch d_primer/run08_trim_R44_1193R_${BCPF_ID}.log
./d_primer/_demultiplex_R44_by_F_primers.sh $BCPF_ID $BCPF >> d_primer/run08_trim_R44_1193R_${BCPF_ID}.log
done < d_primer/_R44_samples.txt &

## R45: De-multiplex samples using bc and primer sequence
## =======================================================================================
while read BCPF_ID BCPF
do
touch d_primer/run08_trim_R45_1193R_${BCPF_ID}.log
./d_primer/_demultiplex_R45_by_F_primers.sh $BCPF_ID $BCPF >> d_primer/run08_trim_R45_1193R_${BCPF_ID}.log
done < d_primer/_R45_samples.txt &

## R46: De-multiplex samples using bc and primer sequence
## =======================================================================================
while read BCPF_ID BCPF
do
touch d_primer/run08_trim_R46_1193R_${BCPF_ID}.log
./d_primer/_demultiplex_R46_by_F_primers.sh $BCPF_ID $BCPF >> d_primer/run08_trim_R46_1193R_${BCPF_ID}.log
done < d_primer/_R46_samples.txt &

### sequence counts
grep -c "@M04679" d_primer/run08_trim_R*_F*.fastq > d_primer/run08_trim_R_and_F_seq_counts.txt

cd ..




## =======================================================================================
## D3 | MiSeq run11
## =======================================================================================
## check if "c_merge/run11_doubled.fastq" needs to be unzipped
# gunzip c_merge/run11_doubled.fastq

mkdir MiSeq_run11/d_primer/
cd MiSeq_run11/

## splitting first by R-primer: 
## =======================================================================================
# cutadapt -a ADAPTER-SEQUENCE input.fastq > output.fastq
# -------------------------------------------------------
# -g ADAPTER, --front = ADAPTER Sequence of an adapter that was ligated to the 5' end.
# -a ADAPTER, --adapter=ADAPTER Sequence of an adapter that was ligated to the 3' end.
# -b ADAPTER, --anywhere=ADAPTER Sequence of an adapter that was ligated to the 5' or 3' end. 

## De-multiplex samples using bc and primer sequence
while read BCPR_ID BCPR
do
rm d_primer/run11_trim_${BCPR_ID}.log
touch d_primer/run11_trim_${BCPR_ID}.log
./d_primer/_demultiplex_by_R_primer.sh $BCPR_ID $BCPR >> d_primer/run11_trim_${BCPR_ID}.log
done < d_primer/_R_primers.txt &

### disk space
gzip c_merge/run11_doubled.fastq &



## splitting then by F-primer the separately "R-splitted" files: 
## =======================================================================================
# cutadapt -a ADAPTER-SEQUENCE input.fastq > output.fastq
# -------------------------------------------------------
# -g ADAPTER, --front = ADAPTER Sequence of an adapter that was ligated to the 5' end.
# -a ADAPTER, --adapter=ADAPTER Sequence of an adapter that was ligated to the 3' end.
# -b ADAPTER, --anywhere=ADAPTER Sequence of an adapter that was ligated to the 5' or 3' end. 


## R7: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R7_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R7_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R7_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &

## R8: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R8_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R8_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R8_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &

## R9: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R9_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R9_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R9_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &

## R10: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R10_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R10_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R10_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &

## R11: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R11_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R11_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R11_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &

## R12: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R12_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R12_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R12_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &

## R13: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R13_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R13_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R13_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &

## R14: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R14_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R14_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R14_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &

## R16: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R16_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R16_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R16_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &

## R18: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R18_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R18_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R18_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &

## R20: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R20_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R20_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R20_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &

## R22: De-multiplex samples using bc and primer sequence
while read BCPF_ID BCPF
do
touch d_primer/run11_trim_R22_ITS2_${BCPF_ID}.log
./d_primer/_demultiplex_R22_by_ITS1f_primers.sh $BCPF_ID $BCPF >> d_primer/run11_trim_R22_ITS2_${BCPF_ID}.log
done < d_primer/_ITS1f_primers.txt &


### sequence counts
grep -c "@M04679" d_primer/run11_trim_R*_F*.fastq > d_primer/__run11_trim_R_and_F_seq_counts.txt &
grep -c "@M04679" d_primer/run11_trim_R*ITS2.fastq > d_primer/__run11_trim_R_its_counts.txt &

cd ..







## =======================================================================================
## =======================================================================================
## E | Quality Filtering - PRINSEQ-lite 0.20.4
## =======================================================================================
## =======================================================================================

mkdir MiSeq_run02/e_qf/
mkdir MiSeq_run08/e_qf/
mkdir MiSeq_run11/e_qf/

for SMPL in `cat MiSeq_run02/e_qf/_run02_sample_list.txt`
do
   prinseq-lite.pl -params MiSeq_run02/e_qf/prinseq.par -fastq MiSeq_run02/d_primer/run02_trim_${SMPL}.fastq -out_good MiSeq_run02/e_qf/run02_trim_${SMPL}_good -out_bad MiSeq_run02/e_qf/run02_trim_${SMPL}_bad -log MiSeq_run02/e_qf/run02_trim_${SMPL}_qc.log
done &

for SMPL in `cat e_qf/_run08_sample_list.txt`
do
   prinseq-lite.pl -params MiSeq_run08/e_qf/prinseq.par -fastq MiSeq_run08/d_primer/run08_trim_${SMPL}.fastq -out_good MiSeq_run08/e_qf/run08_trim_${SMPL}_good -out_bad MiSeq_run08/e_qf/run08_trim_${SMPL}_bad -log MiSeq_run08/e_qf/run08_trim_${SMPL}_qc.log
done &

for SMPL in `cat MiSeq_run11/e_qf/_run11_sample_list.txt`
do
   prinseq-lite.pl -params MiSeq_run11/e_qf/prinseq.par -fastq MiSeq_run11/d_primer/run11_trim_${SMPL}.fastq -out_good MiSeq_run11/e_qf/run11_trim_${SMPL}_good -out_bad MiSeq_run11/e_qf/run11_trim_${SMPL}_bad -log MiSeq_run11/e_qf/run11_trim_${SMPL}_qc.log
done &


## The parameter file contains:
# out_format 1
# # range_len 350-400, not done
# range_gc 30-70
# min_qual_mean 20
# ns_max_n 0
# noniupac
# lc_method dust
# lc_threshold 15

### sequence counts
grep -c ">" MiSeq_run02/e_qf/run02_trim_*_good.fasta > MiSeq_run02/e_qf/run02_trimmed_and_qc_seq_counts.txt
grep -c ">" MiSeq_run08/e_qf/run08_trim_*_good.fasta > MiSeq_run08/e_qf/run08_trimmed_and_qc_seq_counts.txt
grep -c ">" MiSeq_run11/e_qf/run11_trim_*_good.fasta > MiSeq_run11/e_qf/run11_trimmed_and_qc_seq_counts.txt

## Add barcode label to reads
for SMPL in `cat MiSeq_run02/e_qf/_run02_sample_list.txt`
do
   awk -v SMPL=${SMPL} '{if($1~">") print $1";barcodelabel="SMPL";";else print $1}' MiSeq_run02/e_qf/run02_trim_${SMPL}_good.fasta > MiSeq_run02/e_qf/run02_trim_${SMPL}_good_renamed.fasta
done &

for SMPL in `cat MiSeq_run08/e_qf/_run08_sample_list.txt`
do
   awk -v SMPL=${SMPL} '{if($1~">") print $1";barcodelabel="SMPL";";else print $1}' MiSeq_run08/e_qf/run08_trim_${SMPL}_good.fasta > MiSeq_run08/e_qf/run08_trim_${SMPL}_good_renamed.fasta
done &

for SMPL in `cat MiSeq_run11/e_qf/_run11_sample_list.txt`
do
   awk -v SMPL=${SMPL} '{if($1~">") print $1";barcodelabel="SMPL";";else print $1}' MiSeq_run11/e_qf/run11_trim_${SMPL}_good.fasta > MiSeq_run11/e_qf/run11_trim_${SMPL}_good_renamed.fasta
done &



### combining samples per experiments
## =======================================================================================

mkdir f_otu_16S

## MiSeq_run02: 
## all maize samples according to _run02_sample_list.txt 
mkdir MiSeq_run02/e_qf_field

for i in {1..44}
do
f=$(head -$i MiSeq_run02/e_qf/_run02_sample_list.txt | tail -1)
cp MiSeq_run02/e_qf/*$f*_renamed.fasta MiSeq_run02/e_qf_field/
done

cat MiSeq_run02/e_qf_field/*_renamed.fasta > f_otu_16S/field_samples_trimmed_qfiltered_renamed.fasta
grep -c ">" f_otu_16S/field_samples_trimmed_qfiltered_renamed.fasta          # 2194037


## MiSeq_run08: 
## all maize samples according to _run08_sample_list.txt 
mkdir MiSeq_run08/e_qf_feedback

for i in {1..40}
do
f=$(head -$i MiSeq_run08/e_qf/_run08_sample_list.txt | tail -1)
cp MiSeq_run08/e_qf/*$f*_renamed.fasta MiSeq_run08/e_qf_feedback/
done

cat MiSeq_run08/e_qf_feedback/*_renamed.fasta > f_otu_16S/feedback_samples_trimmed_qfiltered_renamed.fasta
grep -c ">" f_otu_16S/feedback_samples_trimmed_qfiltered_renamed.fasta          # 6103721




mkdir f_otu_its

## MiSeq_run11: 
## all maize samples according to _run11_sample_list.txt (taken from Database S1 of the MS)

for i in {1..84}
do
f=$(head -$i MiSeq_run11/e_qf/_run11_sample_list.txt | tail -1)
cp MiSeq_run11/e_qf/*$f*_renamed.fasta MiSeq_run11/f_otu_its/
done

cat MiSeq_run11/f_otu_its/*F_good_renamed.fasta > MiSeq_run11/f_otu_its/its_all_samples_trimmed_qfiltered_renamed.fasta
grep -c ">" MiSeq_run11/f_otu_its/its_all_samples_trimmed_qfiltered_renamed.fasta # 2885710




### disk space
## =======================================================================================

# Remove the *trimmed* fastq files in d_primer/ - these files are not needed any longer
rm MiSeq_run02/d_primer/run02_trim*.info
rm MiSeq_run02/d_primer/run02_trim*.fastq
rm MiSeq_run08/d_primer/run08_trim*.info
rm MiSeq_run08/d_primer/run08_trim*.fastq
rm MiSeq_run11/d_primer/run11_trim*.info
rm MiSeq_run11/d_primer/run11_trim*.fastq

# Remove the *bad.fasta and *good.fasta files in e_qf/ - these files are not needed any longer
rm MiSeq_run02/e_qf/run02_trim*bad.fasta
rm MiSeq_run02/e_qf/run02_trim*good.fasta
rm MiSeq_run08/e_qf/run08_trim*bad.fasta
rm MiSeq_run08/e_qf/run08_trim*good.fasta
rm MiSeq_run11/e_qf/run11_trim*bad.fasta
rm MiSeq_run11/e_qf/run11_trim*good.fasta

# compress the demultiplexed, quality filtered and renamed sequences in e_qf
gzip MiSeq_run02/e_qf/run02_trim_*renamed.fasta &
gzip MiSeq_run08/e_qf/run08_trim_*renamed.fasta &
gzip MiSeq_run11/e_qf/run11_trim_*renamed.fasta &





## =======================================================================================
## =======================================================================================
## F | 16S OTU clustering  (run02 and run08 together)
## =======================================================================================
## =======================================================================================

ls -l f_otu_16S
grep -c ">" f_otu_16S/*_renamed.fasta

# f_otu_16S/feedback_samples_trimmed_qfiltered_renamed.fasta:6103721  # (feedback experiment; run08)
# f_otu_16S/field_samples_trimmed_qfiltered_renamed.fasta:2194037     # (field experiment; run02)

cat f_otu_16S/field_samples_trimmed_qfiltered_renamed.fasta f_otu_16S/feedback_samples_trimmed_qfiltered_renamed.fasta > f_otu_16S/both_exps_trimmed_qfiltered_renamed.fasta

grep -c ">" f_otu_16S/both_exps_trimmed_qfiltered_renamed.fasta       # 8297758

cd f_otu_16S

##### sequence numbers and length distribution (takes 15 minutes)
mothur "#summary.seqs(fasta=both_exps_trimmed_qfiltered_renamed.fasta)"

#		Start	End	NBases	Ambigs	Polymer	NumSeqs
#Minimum:	1	6	6	0	1	1
#2.5%-tile:	1	359	359	0	4	207444
#25%-tile:	1	368	368	0	5	2074440
#Median: 	1	376	376	0	5	4148880
#75%-tile:	1	379	379	0	5	6223319
#97.5%-tile:	1	385	385	0	6	8090315
#Maximum:	1	494	494	0	56	8297758
#Mean:	1	373.501	373.501	0	5.13854
# of Seqs:	8297758
#Output File Names: 
#both_exps_trimmed_qfiltered_renamed.summary
#[WARNING]: your sequence names contained ':'.  I changed them to '_' to avoid problems in your downstream analysis.



##### OTU clustering

### UPARSE/USEARCH - Workflow for usearch 
usearch -help
usearch -version    # usearch8.1.1812_i86linux64

U="/yourpath/usearch8.1.1812_i86linux64"    # U="usearch"

## sort reads by abundance and trim to common length
${U} -fastx_truncate both_exps_trimmed_qfiltered_renamed.fasta -trunclen 360 -label_suffix _l360 -fastaout both_exps_trimmed_qfiltered_renamed_l360.fasta
# 226373 (2.7%) too short
${U} -sortbylength both_exps_trimmed_qfiltered_renamed_l360.fasta -fastaout both_exps_trimmed_qfiltered_renamed_l360_sort.fasta
# local macbook: out of memory

## dereplicate reads
${U} -derep_fulllength both_exps_trimmed_qfiltered_renamed_l360_sort.fasta -fastaout both_exps_trimmed_qfiltered_renamed_l360_sort_derep.fasta -sizeout
# 8071385 seqs, 4367425 uniques, 4033196 singletons (92.3%) 
# on the server: Memory limit of 32-bit process exceeded, 64-bit build required
grep -c ">" both_exps_trimmed_qfiltered_renamed_l360_sort_derep.fasta
# 4367425

## sort by abundance size 
${U} -sortbysize both_exps_trimmed_qfiltered_renamed_l360_sort_derep.fasta -fastaout both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5.fasta -minsize 5
grep -c ">" both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5.fasta
# 83013

## cluster OTUs (default)
${U} -cluster_otus both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5.fasta -otus both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5_otu.fasta -relabel OTU
4140 OTUs, 9061 chimeras (10.9%)

## Chimera removal
REFdb="/yourpath/gold.fa"
${U} -uchime_ref both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5_otu.fasta -db ${REFdb} -nonchimeras both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5_otu_chimerafree.fasta -chimeras both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5_otu_chimera.fasta -strand plus -threads 10 &
00:00  44Mb  100.0% Reading both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5_otu.fasta
00:01  65Mb  100.0% Reading /yourpath/gold.fa                       
00:01  32Mb  100.0% Masking                                           
00:02  33Mb  100.0% Word stats
00:02  33Mb  100.0% Alloc rows
00:03  89Mb  100.0% Build index
00:06 805Mb  100.0% Found 104/4140 chimeras (2.5%), 3187 not classified (77.0%)
00:06 805Mb  100.0% Writing 104 chimeras                                       
00:06 805Mb  100.0% Writing 4036 non-chimeras and unclassifieds

## mapping the reads to the OTUs (id 0.97)
${U} -usearch_global  both_exps_trimmed_qfiltered_renamed_l360_sort.fasta -db both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5_otu_chimerafree.fasta -strand plus -id 0.97 -uc both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5_otu_chimerafree.uc
70.0% matched

## convert uc-2-tab
python /yourpath/python_scripts_edgar/uc2otutab.py both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5_otu_chimerafree.uc > both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5_otu_chimerafree.tab &


### taxonomy
# By default, QIIME uses the RDP classifier to assign taxonomic data to each representative sequence
# some options
# -c Minimum confidence to record an assignment [default: 0.8]
# took about 1h
## Taxonomy assignment with RDP classifier in QIIME
source /usr/local/QIIME-1.8.0/activate.sh
TAXdb="/yourpath/Silva119_release/taxonomy/97/taxonomy_97_7_levels.txt"
REFSEQ="/yourpath/Silva119_release/rep_set/97/Silva_119_rep_set97.fna"

assign_taxonomy.py -i both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5_otu_chimerafree.fasta -o SILVA/ -t ${TAXdb} -r ${REFSEQ} -c 0.5 -m rdp --rdp_max_memory 32000 &

## re-formatting the taxonomy file for import into R
# code taken from https://stat.ethz.ch/pipermail/r-help/2008-April/159957.html

cd SILVA/ 
awk '
  BEGIN{FS="[\t]|[;]"; OFS="\t"}; {$1=$1}; {print $0}
' < both_exps_trimmed_qfiltered_renamed_l360_sort_derep_ab5_otu_chimerafree_tax_assignments.txt  > SILVA_tax_forR.txt
cd ../






## =======================================================================================
## =======================================================================================
## F | ITS OTU clustering  (run11)
## =======================================================================================
## =======================================================================================

cd MiSeq_run11/
ls -l f_otu_its
grep -c ">" f_otu_its/*_renamed.fasta

# f_otu_its/its_all_samples_trimmed_qfiltered_renamed.fasta:2885710  # (field and feedback experiment; run11)

cd f_otu_its

##### sequence numbers and length distribution (takes 15 minutes)
mothur "#summary.seqs(fasta=its_all_samples_trimmed_qfiltered_renamed.fasta)"

#                Start   End     NBases  Ambigs  Polymer NumSeqs
#Minimum:        1       9       9       0       2       1
#2.5%-tile:      1       181     181     0       4       72143
#25%-tile:       1       218     218     0       4       721428
#Median:         1       226     226     0       5       1442856
#75%-tile:       1       243     243     0       6       2164283
#97.5%-tile:     1       386     386     0       9       2813568
#Maximum:        1       481     481     0       25      2885710
#Mean:   1       238.533 238.533 0       5.24052
# of Seqs:      2885710
#Output File Names: 
#its_all_samples_trimmed_qfiltered_renamed.summary
#[WARNING]: your sequence names contained ':'.  I changed them to '_' to avoid problems in your downstream analysis.




##### OTU clustering

### UPARSE/USEARCH - Workflow for usearch 
usearch -help
usearch -version    # usearch8.1.1812_i86linux64

ls /tmp/kschlaep/
# usearch8.1.1812_i86linux64

U="/tmp/kschlaep/usearch8.1.1861_i86linux64"
# U="usearch"


## dereplicate reads
${U} -derep_fulllength its_all_samples_trimmed_qfiltered_renamed.fasta -fastaout its_all_samples_trimmed_qfiltered_renamed_derep.fasta -sizeout
# 2885710 seqs, 390590 uniques, 305126 singletons (78.1%)  
grep -c ">" its_all_samples_trimmed_qfiltered_renamed_derep.fasta
# 390590

## sort by abundance size 
${U} -sortbysize its_all_samples_trimmed_qfiltered_renamed_derep.fasta -fastaout its_all_samples_trimmed_qfiltered_renamed_derep_ab5.fasta -minsize 5
grep -c ">" its_all_samples_trimmed_qfiltered_renamed_derep_ab5.fasta
# 28685

## cluster OTUs (default)
${U} -cluster_otus its_all_samples_trimmed_qfiltered_renamed_derep_ab5.fasta -otus its_all_samples_trimmed_qfiltered_renamed_derep_ab5_otu.fasta -relabel OTU
# 2302 OTUs, 685 chimeras (2.4%)

## mapping the reads to the OTUs (id 0.97)
${U} -usearch_global its_all_samples_trimmed_qfiltered_renamed.fasta -db its_all_samples_trimmed_qfiltered_renamed_derep_ab5_otu.fasta -strand plus -id 0.97 -uc its_all_samples_trimmed_qfiltered_renamed_derep_ab5_otu.uc -otutabout its_all_samples_trimmed_qfiltered_renamed_derep_ab5_otu.tab
# 97.9% matched


### taxonomy
# By default, QIIME uses the RDP classifier to assign taxonomic data to each representative sequence
# some options
# -c Minimum confidence to record an assignment [default: 0.8]
# took about 1h
## Taxonomy assignment with RDP classifier in QIIME
source /usr/local/QIIME-1.8.0/activate.sh

TAXdb="/yourpath/UNITE/sh_taxonomy_qiime_ver7_dynamic_s_28.06.2017.txt"
REFSEQ="/yourpath/UNITE/sh_refs_qiime_ver7_dynamic_s_28.06.2017.fasta"

assign_taxonomy.py -i its_all_samples_trimmed_qfiltered_renamed_derep_ab5_otu.fasta -o UNITE/ -t ${TAXdb} -r ${REFSEQ} -c 0.5 -m rdp --rdp_max_memory 32000 &


## re-formatting the taxonomy file for import into R
# code taken from https://stat.ethz.ch/pipermail/r-help/2008-April/159957.html

cd UNITE/ 
awk '
  BEGIN{FS="[\t]|[;]"; OFS="\t"}; {$1=$1}; {print $0}
' < its_all_samples_trimmed_qfiltered_renamed_derep_ab5_otu_tax_assignments.txt  > UNITE_tax_forR.txt
cd ../


